package gods.bubble.index;

import gods.bubble.common.StandardDocument;
import gods.bubble.parser.FileParser;
import gods.bubble.util.Config;
import gods.bubble.util.SimpleFileIOUtil;

import java.io.BufferedReader;
import java.io.File;
import java.io.FileReader;
import java.io.IOException;
import java.util.Date;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.document.Document;
import org.apache.lucene.document.Field;
import org.apache.lucene.index.IndexWriter;
import org.apache.lucene.store.Directory;
import org.apache.lucene.store.FSDirectory;
import org.wltea.analyzer.lucene.IKAnalyzer;

//Each record has fields as follow:
//	title, stored, analyzed
//	contents, stored(html)/unstored(pdf, ms document), analyzed
//	url, stored, unanalyzed

public class Indexer {
	public void index(String logFile, String indexDir)
			throws IOException {
		System.out.println(new Date() + "--Start Indexing...");
		File logData = new File(logFile);

		BufferedReader reader = new BufferedReader(new FileReader(logData));

		Directory directory = FSDirectory.open(new File(indexDir));
		Analyzer analyzer = new IKAnalyzer();

		IndexWriter writer = new IndexWriter(directory, analyzer, true,
				IndexWriter.MaxFieldLength.LIMITED);

		String line = null;

		int num = 0;
		int scholar_num = 0;

		while ((line = reader.readLine()) != null) {
			String[] args = line.split("\t");

			String title = args[0];
			String url = args[1];
			double pagerank = Double.valueOf(args[2]);

			// For scholar search
			if (!title.equals("No Title Document")) {
				continue;
			}

			StandardDocument sd = FileParser.parser(url);

			if (sd == null) {
				continue;
			}
			
			if (sd.title == null || sd.title.length() < 1) {
				sd.title = title;
			}

			sd.pagerank = pagerank;

			SimpleFileIOUtil.stringToFile(sd.toXml(), new File(Config.obDir
					+ sd.fileName + ".txt"), "UTF-8");

			write(writer, sd);
			num++;
		}

		System.out.println("Usual Document: " + num);
		System.out.println("Scholar Document: " + scholar_num);

		writer.optimize();
		writer.close();

		System.out.println(new Date() + "--Indexing Finished!!!");
	}

	public void write(IndexWriter writer, StandardDocument sd)
			throws IOException {
		Document document = new Document();

		try {
			document.add(new Field("url", sd.url, Field.Store.YES,
					Field.Index.ANALYZED));
			document.add(new Field("title", sd.title, Field.Store.YES,
					Field.Index.ANALYZED));
			document.add(new Field("content", sd.content, Field.Store.YES,
					Field.Index.ANALYZED));
			document.add(new Field("pagerank", Double.toString(sd.pagerank),
					Field.Store.YES, Field.Index.NOT_ANALYZED));

			document.getField("title").setBoost(3);

			writer.addDocument(document);
		} catch (Exception e) {
			System.out.println("Failed in indexing " + sd.title);
		}
	}

	public void indexScholar(String logFile, String scholarIndexDir) throws IOException {
		System.out.println(new Date() + "--Start Indexing...");
		File logData = new File(logFile);
		
		BufferedReader reader = new BufferedReader(new FileReader(logData));
		

		Directory scholarDirectory = FSDirectory.open(new File(scholarIndexDir));
		Analyzer analyzer = new IKAnalyzer();
		IndexWriter scholarWriter = new IndexWriter(scholarDirectory, analyzer, true,
				IndexWriter.MaxFieldLength.LIMITED);
		
		int scholar_num = 0;
		
		String line = null;
		while ((line = reader.readLine()) != null) {
			String[] args = line.split("\t");

			String title = args[0];
			String url = args[1];
			
			// For scholar search
			if (!title.equals("No Title Document") && title.length() > 1) {
				Document d = new Document();
				
				System.out.println("Scholar Indexer : get " + title + "; url = " + url);
				d.add(new Field("url", url, Field.Store.YES, Field.Index.ANALYZED));
				d.add(new Field("title", title, Field.Store.YES, Field.Index.ANALYZED));
				d.add(new Field("content", "", Field.Store.YES, Field.Index.ANALYZED));
				d.add(new Field("pagerank", "0", Field.Store.YES, Field.Index.NOT_ANALYZED));
				
				scholarWriter.addDocument(d);
				
				scholar_num++;
				
			}
		}
			
		System.out.println("Scholar Document: " + scholar_num);

		
		System.out.println(scholarWriter.numDocs());
		scholarWriter.optimize();
		scholarWriter.close();

		System.out.println(new Date() + "--Indexing Finished!!!");
	}

	public static void main(String args[]) {
		Indexer indexer = new Indexer();

		try {
			indexer.index(Config.pagerankFile, Config.indexDir);
			//indexer.indexScholar(Config.pagerankFile, Config.scholarIndexDir);
		} catch (IOException e) {
			e.printStackTrace();
		}
	}
}
